import hashlib
import re
import time
from io import BytesIO
import random
import oss2
import redis
import requests
from PIL import Image
from lxml import etree
import csv
from selenium.webdriver import Chrome
from pymysql import connect
import os
import sys

father_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

sys.path.append(father_path)

from conf.conf import *

# 获取当前文件的父目录
from utils.dingding_msg import send_msg


def get_token():
    token_queue = redis.Redis(host=redis_host, port=redis_port, db=redis_db)
    ret = [i for i in token_queue.smembers('token_cookie')]
    return ret[0].decode()


class MpWebdriver(object):
    def __init__(self):
        self.driver = Chrome()
        pass

    def get_cookie(self):
        self.driver.get('https://mp.weixin.qq.com/cgi-bin/home?t=home/index&lang=zh_CN&token=1841527783')
        pass


class Spider(object):
    def __init__(self, token, cookie_str):
        self.token = token
        self.headers = {
            'accept': 'application/json, text/javascript, */*; q=0.01',
            'accept-encoding': 'gzip, deflate, br',
            'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'sec-fetch-mode': 'cors',
            'sec-fetch-site': 'same-origin',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36',
            'x-requested-with': 'XMLHttpRequest',
            'cookie': cookie_str
        }
        self.local_fakeid_dict = {}
        with open('../conf/mp_bizs', 'r') as f:
            for i in f.readlines():
                try:
                    self.local_fakeid_dict[i.split('***')[0].strip()] = i.split('***')[1].strip()
                except Exception:
                    break
        self.request_num = 0
        self.conn = connect(host=mysql_host, port=mysql_port, user=mysql_user,
                            passwd=mysql_pwd, db=mysql_db, charset='utf8mb4')
        self.mp_not_exist = [i.strip() for i in open('../conf/mp_not_exist', 'r').readlines()]

    def save_item(self, item_list):
        cursor = self.conn.cursor()
        for item in item_list:
            try:
                sql1 = "INSERT INTO recovery_data (title,url,pubtime,content,site_id,group_id,img_status,type,url_md5) " \
                       "VALUES ('%s','%s','%s','%s',%s,%s,%s,3,'%s')" % (
                           item['title'], item['link'], item['pubtime'], item['content'], item['site_id'],
                           item['group_id'],
                           item['img_status'], item['url_md5'])
                self.conn.ping(reconnect=True)
                cursor.execute(sql1)
                print(sql1)
                data_id = self.conn.insert_id()
                for im in item['img_list']:
                    try:
                        sql2 = "INSERT IGNORE INTO recovery_data_img (data_id,img_url,group_id)VALUES (%s,'%s',%s)" % (
                            data_id, im, item['group_id'])
                        self.conn.ping(reconnect=True)
                        cursor.execute(sql2)
                        print(sql2)
                    except Exception as e:
                        print(e)
            except Exception as e:
                self.conn.rollback()
            else:
                self.conn.ping(reconnect=True)
                self.conn.commit()
        cursor.close()

    def gen_session(self):
        token, cookie_str = get_token().split('***')
        print(token, cookie_str)
        self.token = token
        self.headers['cookie'] = cookie_str

    def download_url(self, url):
        while 1:
            try:
                time.sleep(random.randint(1, 3))
                url = re.sub(r'token=(\d+)', 'token=' + self.token, url)
                resp = requests.get(url=url, headers=self.headers)

                if resp.json()['base_resp']['err_msg'] == 'invalid session':
                    send_msg('微信，session过期了' + resp.json()['base_resp']['err_msg'])
                    time.sleep(60)
                    print('session过期')
                    self.gen_session()
                if resp.json()['base_resp']['err_msg'] != 'ok':
                    send_msg('微信，session被限制了，请换token与session' + resp.json()['base_resp']['err_msg'])
                    time.sleep(60)
                    print('error session')
                    self.gen_session()
                else:
                    break
            except Exception:
                print('下载错误')
                time.sleep(2)
        self.request_num += 1
        print(self.request_num)
        return resp

    def get_article_content(self, url):
        resp = requests.get(url)
        ele = etree.HTML(resp.content.decode())
        content = ''.join(ele.xpath('.//div[@class="rich_media_content "]//text()')).strip()
        img_urls = ele.xpath('.//img/@data-src')
        img_status, img_list = self.oss2_img(img_urls)
        return content, img_status, img_list

    def oss2_img(self, imgs):
        img_list = []
        if imgs is not []:
            for img_url in imgs:
                try:
                    headers = {
                        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
                    }
                    inputs = requests.get(img_url, headers=headers)
                    tmpIm = BytesIO(inputs.content)
                    im = Image.open(tmpIm)
                    w = im.size[0]
                    h = im.size[1]
                    if w >= 200 or h >= 200:
                        lost = img_url.split("/mmbiz_")[1].split('/')[0]
                        md = hashlib.md5()  # 构造一个md5
                        pat = str(img_url)
                        md.update(str(pat).encode())
                        img_url_md5 = md.hexdigest()
                        img_key = 'yuqing/10-63/' + str(img_url_md5) + '.' + lost
                        img_urls = 'https://yuwoyg.oss-cn-hangzhou.aliyuncs.com/' + img_key
                        # 阿里云主账号AccessKey拥有所有API的访问权限，风险很高。强烈建议您创建并使用RAM账号进行API访问或日常运维，请登录 https://ram.console.aliyun.com 创建RAM账号。
                        auth = oss2.Auth('LTAIhIK9hoKRcMOy', 'Llx7NO8k6VkHgMvMws7SguFSKhwydI')
                        # Endpoint以杭州为例，其它Region请按实际情况填写。
                        bucket = oss2.Bucket(auth, 'http://oss-cn-hangzhou.aliyuncs.com', 'yuwoyg')
                        # # requests.get返回的是一个可迭代对象（Iterable），此时Python SDK会通过Chunked Encoding方式上传。
                        bucket.put_object(img_key, inputs)
                        img_list.append(img_urls)
                except Exception as e:
                    print('oss:' + str(e))
        if img_list == []:
            img_status = 0
        else:
            img_status = 1
        return img_status, img_list

    def get_detail(self, fakeid, st_time, group_id, site_id):
        flag = 0
        ret_list = []
        pn = 0
        while 1:
            url = 'https://mp.weixin.qq.com/cgi-bin/appmsg?token={}&lang=zh_CN&f=json&ajax=1&random=0.21068279797018374&action=list_ex&begin={}&count={}&query=&fakeid={}&type=9'.format(
                token, pn, pn + 5, fakeid)
            pn += 5
            resp = self.download_url(url).json()
            for i in resp['app_msg_list']:
                if i['create_time'] <= st_time:
                    flag = 1
                    break
                item = {}
                item['title'] = i['title']
                item['link'] = i['link']
                item['content'], item['img_status'], item['img_list'] = self.get_article_content(item['link'])
                item['pubtime'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(i['create_time']))
                item['group_id'] = group_id
                item['site_id'] = site_id
                md = hashlib.md5()  # 构造一个md5
                pat = str(item['pubtime']) + str(item['title']) + str(group_id) + str(site_id)
                md.update(str(pat).encode())
                item['url_md5'] = md.hexdigest()
                print(item)
                ret_list.append(item)
            if flag == 1:
                break
        return ret_list

    def search_wechat(self, wd):
        url = 'https://mp.weixin.qq.com/cgi-bin/searchbiz?action=search_biz&token={}&lang=zh_CN&f=json&ajax=1&random=0.2804958465931593&query={}&begin=0&count=5'.format(
            self.token, wd)
        resp = self.download_url(url).json()
        with open('../conf/mp_bizs', 'a') as f:
            for i in resp['list']:
                if i['nickname'] == wd:
                    f.write(i['nickname'] + '***' + i['fakeid'] + '\n')
                    f.close()
                    return i['fakeid']

    def unchanged_mps(self):
        import time
        TimeStr = "2019-10-29 00:00:00"
        formatStr = "%Y-%m-%d %H:%M:%S"
        tmObject = time.strptime(TimeStr, formatStr)
        tmStamp = time.mktime(tmObject)

        mp_csv = csv.reader(open('../conf/mp_wechat.csv'))
        index = 0
        for mp in mp_csv:
            index += 1
            if index <= 30:
                continue

            group_id = mp[2]
            fakeid = self.local_fakeid_dict.get(mp[1])
            if fakeid == None:
                fakeid = self.search_wechat(mp[1])
            if fakeid == None:
                print('账户不存在:', mp[1])
                open('../conf/mp_not_exist', 'a').write(mp[1] + '\n')
                continue
            site_id = mp[0]
            ret_list = self.get_detail(fakeid=fakeid, st_time=tmStamp, group_id=group_id, site_id=site_id)
            # self.save_item(ret_list)

    def gen_mps_dict(self):
        _sql = '''select rs.id ,rs.name,group_concat(rsg.group_id) as 'group_id'
                            from
                             recovery_site_group rsg,
                             recovery_site rs
                            where rsg.site_id=rs.id and rs.type=3
                            group by rs.sougou_weixin_id
                            ,rs.id;'''
        cursor = self.conn.cursor()
        cursor.execute(_sql)
        ret_list = cursor.fetchall()
        cursor.close()
        mps_list = []
        for i in ret_list:
            mps_dict = {}
            mps_dict['site_id'] = i[0]
            mps_dict['group_id'] = i[2]
            mps_dict['name'] = i[1]
            mps_list.append(mps_dict)
        return mps_list

    def run(self):
        import time
        mp_list = self.gen_mps_dict()
        t = time.localtime(time.time())
        tmStamp_1 = time.mktime(time.strptime(time.strftime('%Y-%m-%d 00:00:00', t), '%Y-%m-%d %H:%M:%S'))
        tmStamp_2 = time.mktime(time.strptime(time.strftime('%Y-%m-%d 08:00:00', t), '%Y-%m-%d %H:%M:%S'))
        tmStamp_3 = time.mktime(time.strptime(time.strftime('%Y-%m-%d 12:00:00', t), '%Y-%m-%d %H:%M:%S'))
        tmStamp_4 = time.mktime(time.strptime(time.strftime('%Y-%m-%d 17:00:00', t), '%Y-%m-%d %H:%M:%S'))
        tmStamp_5 = time.mktime(time.strptime(time.strftime('%Y-%m-%d 19:00:00', t), '%Y-%m-%d %H:%M:%S'))
        tmStamp_6 = time.mktime(time.strptime(time.strftime('%Y-%m-%d 23:00:00', t), '%Y-%m-%d %H:%M:%S'))
        tmStamp_7 = time.mktime(time.strptime(time.strftime('%Y-%m-%d 23:59:59', t), '%Y-%m-%d %H:%M:%S'))
        time_list = [tmStamp_2, tmStamp_3, tmStamp_4, tmStamp_5, tmStamp_6, tmStamp_7]
        pn = 0
        while 1:
            t = time_list[pn]
            if time.time() < t:
                time.sleep(60)
            else:
                pn += 1
            for mp in mp_list:
                group_id = mp['group_id']
                if mp['name'] in self.mp_not_exist:
                    continue
                fakeid = self.local_fakeid_dict.get(mp['name'])
                if fakeid == None:
                    fakeid = self.search_wechat(mp['name'])
                if fakeid == None:
                    print('账户不存在:', mp['name'])
                    open('../conf/mp_not_exist', 'a').write(mp['name'] + '\n')
                    continue
                site_id = mp['site_id']
                ret_list = self.get_detail(fakeid=fakeid, st_time=tmStamp_1, group_id=group_id, site_id=site_id)
                if ret_list != []:
                    mp_list.remove(mp)
                self.save_item(ret_list)


if __name__ == '__main__':
    token, cookie_str = get_token().split('***')
    print(token, cookie_str)
    s = Spider(token, cookie_str)
    while 1:
        s.run()
